"""
预处理数据，保留主干,10000数据交给一个进程处理
"""
from multiprocessing import Pool
import jieba.posseg
import re
import threading

keep = ["n", "f", "s", "nr", "ns", "nt", "nw", "nz",
        "v", "vd", "vn", "t", "m", "q", "TIME", "ORG", "LOC", "PER", "w"]
set_not = ["不", "非", "有", "无", "莫", "高度"]

jieba.enable_paddle()  # 开启paddle模式
lock = threading.Lock()  # 申请一把锁


def keep_main_content(line):
    set_l = line.strip().split("\t")
    if bool(re.search("[-\d()*+/=\.]{15,}", set_l[1])):
        return
    words = jieba.posseg.cut(set_l[1], use_paddle=True)
    source = []
    target = []
    for item in words:
        target.append(item.word)
        if item.flag in keep or item.word in set_not:
            source.append(item.word)
    source = " ".join(source)
    target = " ".join(target)
    return source + "\t" + target + "\n"


def handle(in1, out1):
    fp = open(in1, 'r', encoding='utf-8')
    fout = open(out1, 'a', encoding='utf-8')
    pool = Pool(processes=4)
    lines = fp.readlines()
    a = 10000
    b = 0
    count = 0
    data = []
    while count < 10 or len(data) > 100:
        data = []
        for i, rows in enumerate(lines):  # 198W行的数据
            if i in range(0 + b, 10000 + b):  # 指定数据哪几行
                data.append(rows)
        b += a
        count += 1
        result = pool.map(keep_main_content, data)
        print('执行到了第几个块：', count)
        print(len(result))
        lock.acquire()  # 加锁
        print(type(result))
        fout.writelines(result)
        lock.release()  # 释放锁
    pool.close()
    pool.join()
    print('sub-process done')


if __name__ == "__main__":
    # infile = '../out3.tsv'
    # outfile = '../keep_content_split.tsv'
    infile = 'd:/1.txt'
    outfile = 'd:/keep_content_bottom.tsv'
    handle(infile, outfile)
